﻿using System;
using System.Linq;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;

using Microsoft.Scripting;
using Microsoft.Scripting.Runtime;
using Microsoft.Scripting.Utils;

using Scala;
using Scala.Compiler;
using Scala.Runtime;
using Scala.Compiler.Ast;

#if CLR2
using Microsoft.Scripting.Math
#else
using System.Numerics;
using System.Collections;
#endif

namespace Scala.Compiler
{
    public class Parser : IDisposable
    {

        private readonly Tokenizer _tokenizer;

        private ErrorSink _errors;
        private ParserSink _sink;

        private SourceUnit _sourceUnit;

        //private ModuleOptions _languageFeatures;

        //state:
        private TokenWithSpan _token;
        private TokenWithSpan _lookahead;
        //private Stack<DefDefStatement> _functions;
        private bool _fromfutureAllowed;
        private string _privatePrefix;
        private bool _parsingStarted, _allowIncomplete;
        private bool _inLoop, _inFinally, _isGenerator, _returnWithValue;
        private SourceCodeReader _sourceReader;
        private int _errorCode;
        private readonly CompilerContext _context;
        private ScalaAst _globalParent;

        private static readonly char[] newLineChar = new char[] { '\n' };
        private static readonly char[] whiteSpace = { ' ', '\t' };

        #region Construction

        private Parser(CompilerContext context, Tokenizer tokenizer, ErrorSink errorSink, ParserSink parserSink)
        {
            ContractUtils.RequiresNotNull(tokenizer, "tokenizer");
            ContractUtils.RequiresNotNull(errorSink, "errorSink");
            ContractUtils.RequiresNotNull(parserSink, "parserSink");

            tokenizer.ErrorSink = new TokenizerErrorSink(this);

            _tokenizer = tokenizer;
            _errors = errorSink;
            if (parserSink != ParserSink.Null)
            {
                _sink = parserSink;
            }
            _context = context;
            Reset(tokenizer.SourceUnit);
        }

        public static Parser CreateParser(CompilerContext context, ScalaOptions options, string filePath)
        {
            return CreateParserWorker(context, options, false, filePath);
        }

        private static Parser CreateParserWorker(CompilerContext context, ScalaOptions options, bool verbatim, string filePath)
        {
            //ContractUtils.RequiresNotNull(context, "context");
            //ContractUtils.RequiresNotNull(options, "options");

            //ScalaCompilerOptions compilerOptions = context.Options as ScalaCompilerOptions;
            //if (options == null)
            //{
            //    throw new Exception("Context is required");
            //}
            SourceCodeReader reader;
            try
            {
                var res = new StreamReader(filePath);
                reader = new SourceCodeReader(res, null);
                //if (compilerOptions.SkipFirstLine)
                //{
                //    reader.ReadLine();
                //}
            }
            catch (IOException e)
            {
                //context.Errors.Add(context.SourceUnit, e.Message, SourceSpan.Invalid, 0, Severity.Error);
                throw;
            }
            var s = ErrorSink.Default;
            Tokenizer tokenizer = new Tokenizer(s);

            tokenizer.Initialize(null, reader, null, SourceLocation.MinValue);

            Parser result = new Parser(null, tokenizer, s, ParserSink.Null);
            result._sourceReader = reader;
            return result;
        }

        #endregion

        #region Public Parser Interface

        public ScalaAst ParseFile(bool makeModule)
        {
            return ParseFile(makeModule, false);
        }

        //single_input: Newline | simple_stmt | compound_stmt Newline
        //eval_input: testlist Newline* ENDMARKER
        //file_input: (Newline | stmt)* ENDMARKER
        public ScalaAst ParseFile(bool makeModule, bool returnValue)
        {
            try
            {
                return ParseFileWorker(makeModule, returnValue);
            }
            catch (BadSourceException bse)
            {
                throw BadSourceError(bse);
            }
        }

        //[stmt_list] Newline | compound_stmt Newline
        //stmt_list ::= simple_stmt (";" simple_stmt)* [";"]
        //compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
        //Returns a simple or coumpound_stmt or null if input is incomplete
        /// <summary>
        /// Parse one or more lines of interactive input
        /// </summary>
        /// <returns>null if input is not yet valid but could be with more lines</returns>
        public ScalaAst ParseInteractiveCode(out ScriptCodeParseResult properties)
        {
            bool parsingMultiLineCmpdStmt;
            bool isEmptyStmt = false;

            properties = ScriptCodeParseResult.Complete;
            _globalParent = new ScalaAst("asdf", null);
            StartParsing();
            Statement ret = InternalParseInteractiveInput(out parsingMultiLineCmpdStmt, out isEmptyStmt);

            if (_errorCode == 0)
            {
                if (isEmptyStmt)
                {
                    properties = ScriptCodeParseResult.Empty;
                }
                else if (parsingMultiLineCmpdStmt)
                {
                    properties = ScriptCodeParseResult.IncompleteStatement;
                }

                if (isEmptyStmt)
                {
                    return null;
                }

                return FinishParsing(ret);
            }
            else
            {
                if ((_errorCode & ErrorCodes.IncompleteMask) != 0)
                {
                    if ((_errorCode & ErrorCodes.IncompleteToken) != 0)
                    {
                        properties = ScriptCodeParseResult.IncompleteToken;
                        return null;
                    }

                    if ((_errorCode & ErrorCodes.IncompleteStatement) != 0)
                    {
                        if (parsingMultiLineCmpdStmt)
                        {
                            properties = ScriptCodeParseResult.IncompleteStatement;
                        }
                        else
                        {
                            properties = ScriptCodeParseResult.IncompleteToken;
                        }
                        return null;
                    }
                }

                properties = ScriptCodeParseResult.Invalid;
                return null;
            }
        }

        private ScalaAst FinishParsing(Statement ret)
        {
            var res = _globalParent;
            _globalParent = null;
            var lineLocs = _tokenizer.GetLineLocations();
            // update line mapping
            //if (_sourceUnit.HasLineMapping)
            //{
            //    List<int> newLineMapping = new List<int>();
            //    int last = 0;
            //    for (int i = 0; i < lineLocs.Length; i++)
            //    {
            //        int newLine = _sourceUnit.MapLine(i + 1) - 1;
            //        while (newLineMapping.Count < i)
            //        {
            //            newLineMapping.Add(last);
            //        }
            //        last = lineLocs[i] + 1;
            //        newLineMapping.Add(lineLocs[i]);
            //    }

            //    lineLocs = newLineMapping.ToArray();
            //}
            res.ParsingFinished(lineLocs, ret);

            return res;
        }

        public ScalaAst ParseSingleStatement()
        {
            try
            {
                _globalParent = new ScalaAst("asdf", null);
                StartParsing();

                MaybeEatNewLine();
                Statement statement = ParseStmt();
                EatEndOfInput();
                return FinishParsing(statement);
            }
            catch (BadSourceException bse)
            {
                throw BadSourceError(bse);
            }
        }

        private Statement ParseStmt()
        {
            switch (PeekToken().Kind)
            {
                case TokenKind.IF:
                    return ParseIfStmt();
                case TokenKind.WHILE:
                    return ParseWhileStmt();
                case TokenKind.FOR:
                    return ParseForStmt();
                case TokenKind.TRY:
                    return ParseTryStatement();
                case TokenKind.AT:
                    return ParseDecorated();
                case TokenKind.DEF:
                    return ParseFuncDef();
                case TokenKind.CLASS:
                    return ParseClassDef();
                case TokenKind.WITH:
                    return ParseWithStmt();
                case TokenKind.OBJECT:
                    return ParseObjectDef();
                default:
                    return ParseSimpleStmt();
            }
        }

        private Statement ParseObjectDef()
        {
            Eat(TokenKind.OBJECT);

            var start = GetStart();
            string name = ReadName();
            Statement body = null;
            if (name == null)
            {
                return new ModuleDefStatement(null, new Expression[0], ErrorStmt());
            }

            Expression[] bases = new Expression[0];
            if (MaybeEat(TokenKind.LPAREN))
            {
                ReportSyntaxError("Object can't have parameters");
            }
            if (MaybeEat(TokenKind.EXTENDS))
            {
                Token baseClass = NextToken();
                if (!(baseClass.Kind != TokenKind.Name))
                {
                    ReportSyntaxError("The base class is not defined properly");
                }
                bases[0] = new NameExpression((string)baseClass.Value);
            }
            var mid = GetEnd();
            if (MaybeEat(TokenKind.LBRACE))
            {
                body = ParseClassOrFuncBody();
            }
            ModuleDefStatement ret = new ModuleDefStatement(name, bases, body);
            ret.HeaderIndex = mid;
            ret.SetLoc(_globalParent, start, GetEnd());
            return ret;
        }

        private Statement ParseWithStmt()
        {
            throw new NotImplementedException();
        }

        private ClassDefStatement ParseClassDef()
        {
            Eat(TokenKind.CLASS);

            var start = GetStart();
            string name = ReadName();
            Statement body = null;
            if (name == null)
            {
                return new ClassDefStatement(null, new Expression[0], ErrorStmt());
            }

            Expression[] bases = new Expression[0];
            if (MaybeEat(TokenKind.LPAREN))
            {
                ReportSyntaxError("Object can't have parameters");
            }
            if (MaybeEat(TokenKind.EXTENDS))
            {
                Token baseClass = NextToken();
                if (!(baseClass.Kind != TokenKind.Name))
                {
                    ReportSyntaxError("The base class is not defined properly");
                }
                bases[0] = new NameExpression((string)baseClass.Value);
            }
            var mid = GetEnd();
            if (MaybeEat(TokenKind.LBRACE))
            {
                body = ParseClassOrFuncBody();
            }
            if (body == null)
            {
                Statement[] sts = new Statement[1];
                sts[0] = new EmptyStatement();
                body = new BlockStatement(sts, new EmptyStatement());
            }
            ClassDefStatement ret = new ClassDefStatement(name, bases, body);
            ret.HeaderIndex = mid;
            ret.SetLoc(_globalParent, start, GetEnd());
            return ret;
        }

        private Statement ParseFuncDef()
        {
            Eat(TokenKind.DEF);
            var start = GetStart();
            var name = ReadName();
            IList<Parameter[]> parameters = null;
            if (PeekToken().Kind == TokenKind.LPAREN)
            {
                parameters = ParseVarArgsList(TokenKind.COLON);
            };

            var lStart = GetStart();
            var lEnd = GetEnd();
            int grouping = _tokenizer.GroupingLevel;

            DefDefStatement ret = null;
            if (parameters == null)
            {
                ret = new DefDefStatement(name, new List<Parameter[]>());
                ret.SetLoc(_globalParent, start, lEnd);
            }

            var rStart = GetStart();
            var rEnd = GetEnd();

            if (ret == null)
                ret = new DefDefStatement(name, parameters);

            if (MaybeEat(TokenKind.COLON))
            {
                var retTypeName = ReadName();
                ret.TypeName = retTypeName;
            }
            //PushFunction(ret);
            Statement body = null;
            if (MaybeEat(TokenKind.EQUALS))
            {
                if (PeekToken().Kind == TokenKind.LBRACE)
                {
                    body = ParseClassOrFuncBody();
                }
                body = ParseSimpleStmt();
            }
            else if (MaybeEat(TokenKind.LBRACE))
            {
                body = ParseClassOrFuncBody();
            }
            else
            {
                ReportSyntaxError("Class not defined properly");
            }

            //DefDefStatement ret2 = PopFunction();
            //System.Diagnostics.Debug.Assert(ret == ret2);

            if (body == null) throw new Exception("Undefined Body of the Object");
            ret.Body = body;
            ret.HeaderIndex = rEnd;
            if (_sink != null)
            {
                _sink.MatchPair(
                    new SourceSpan(_tokenizer.IndexToLocation(lStart), _tokenizer.IndexToLocation(lEnd)),
                    new SourceSpan(_tokenizer.IndexToLocation(rStart), _tokenizer.IndexToLocation(rEnd)),
                    grouping
                   );
            }
            ret.SetLoc(_globalParent, start, body.EndIndex);
            return ret;
        }

        private IList<Parameter[]> ParseVarArgsList(TokenKind terminator)
        {
            List<List<Parameter>> p = new List<List<Parameter>>();
            Eat(TokenKind.LPAREN);
            List<Parameter> pl = new List<Parameter>();
            HashSet<string> names = new HashSet<string>(StringComparer.Ordinal);
            bool needDefault = false;
            for (int position = 0; ; position++)
            {
                if (PeekToken().Kind == terminator) break;
                if (PeekToken().Kind == TokenKind.LBRACE) break;
                if (PeekToken().Kind == TokenKind.EQUALS) break;
                Parameter parameter;

                if ((parameter = ParseParameter(position, names)) != null)
                {
                    pl.Add(parameter);
                    if (MaybeEat(TokenKind.EQUALS))
                    {
                        needDefault = true;
                        parameter.DefaultValue = ParseExpression();
                    }
                    else if (needDefault)
                    {
                        ReportSyntaxError("default Value must be specified here");
                    }
                }
                else
                {
                    return null;
                }

                if (MaybeEat(TokenKind.RPAREN))
                {
                    p.Add(pl);
                }
                if (MaybeEat(TokenKind.LPAREN))
                {
                    pl = new List<Parameter>();
                }
                MaybeEat(TokenKind.COMMA);
            }
            var res = from x in p select x.ToArray();
            return res.ToList();
        }

        private Parameter ParseParameter(int position, HashSet<string> names)
        {
            Token t = PeekToken();
            Parameter parameter = null;

            switch (t.Kind)
            {
                case TokenKind.Name:
                    NextToken();
                    string name = (string)t.Value;
                    Eat(TokenKind.COLON);
                    Token typeNameToken = NextToken();
                    if (!(typeNameToken.Kind == TokenKind.Name))
                    {
                        ReportSyntaxError("The Return Type of the function is not mentioned correctly");
                    }
                    string typeName = (string)typeNameToken.Value;
                    if (MaybeEat(TokenKind.ARROW))
                    {
                        parameter = new Parameter(name, ParameterKind.Function);
                        Token retTypeNameToken = NextToken();
                        if (!(retTypeNameToken.Kind == TokenKind.Name))
                        {
                            ReportSyntaxError("The Return Type of the function is not mentioned correctly");
                        }
                        string retTypeName = (string)retTypeNameToken.Value;
                        parameter.ReturnType = retTypeName;
                        parameter.TypeName = typeName;
                    }
                    else
                    {
                        parameter = new Parameter(name);
                        parameter.TypeName = typeName;
                    }
                    CompleteParameterName(parameter, name, names);
                    break;
                default:
                    ReportSyntaxError(_lookahead);
                    break;
            }
            return parameter;
        }

        private void CompleteParameterName(Tree node, string name, HashSet<string> names)
        {
            if (_sink != null)
            {
                _sink.StartName(GetSourceSpan(), name);
            }
            CheckUniqueParameter(names, name);
            node.SetLoc(_globalParent, GetStart(), GetEnd());
        }

        private Parameter ParseParameterName(HashSet<string> names, ParameterKind kind)
        {
            string name = ReadName();
            if (name != null)
            {
                CheckUniqueParameter(names, name);
            }
            else
            {
                return null;
            }
            Parameter parameter = new Parameter(name, kind);
            parameter.SetLoc(_globalParent, GetStart(), GetEnd());
            return parameter;
        }

        private void CheckUniqueParameter(HashSet<string> names, string name)
        {
            if (names.Contains(name))
            {
                ReportSyntaxError(String.Format(System.Globalization.CultureInfo.InvariantCulture,
                    Resources.DuplicateArgumentInFuncDef,
                    name
                    ));
            }
            names.Add(name);
        }

        private Statement ParseClassOrFuncBody()
        {
            Statement body;
            bool inLoop = _inLoop, inFinally = _inFinally, returnWithValue = _returnWithValue;
            try
            {
                _inLoop = false;
                _inFinally = false;
                _returnWithValue = false;
                body = ParseBlock();
            }
            finally
            {
                _inLoop = inLoop;
                _inFinally = inFinally;
                _returnWithValue = returnWithValue;
            }
            return body;
        }

        private Statement ParseBlock()
        {
            //if(!EatNoEof(TokenKind.EQUALS))
            //{
            //    return ErrorStmt();
            //}

            TokenWithSpan cur = _lookahead;
            List<Statement> l = new List<Statement>();

            while (true)
            {
                if (MaybeEat(TokenKind.RBRACE)) break;
                if (PeekToken().Kind == TokenKind.EOF)
                {
                    ReportSyntaxError("unexpected end of File");
                    break;
                }
                Statement s = ParseStmt();
                l.Add(s);
            }
            Statement[] stmts = l.ToArray();
            if (stmts.Length == 1)
            {
                Statement[] sts = new Statement[1];
                sts[0] = stmts[0];
                BlockStatement r = new BlockStatement(sts, new EmptyStatement());
                return r;
            }
            if (stmts.Length == 0)
            {
                Statement[] sts = new Statement[1];
                sts[0] = new EmptyStatement();
                BlockStatement r = new BlockStatement(sts, new EmptyStatement());
                return r;
            }
            Statement[] stats = new Statement[stmts.Length - 1];
            for (int i = 0; i < stmts.Length - 1; i++)
            {
                stats[i] = stmts[i];
            }
            BlockStatement ret = new BlockStatement(stats, stmts[stmts.Length - 1]);
            ret.SetLoc(_globalParent, stmts[0].StartIndex, stmts[stmts.Length - 1].EndIndex);
            return ret;
            //else
            //{
            //    Statement s = ParseStmt();
            //    s.SetLoc(_globalParent, s.StartIndex, s.EndIndex);
            //    return s;
            //}
        }

        private Statement ErrorStmt()
        {
            return new ExpressionStatement(Error());
        }

        private ErrorExpression Error()
        {
            var res = new ErrorExpression();
            res.SetLoc(_globalParent, GetStart(), GetEnd());
            return res;
        }

        //private DefDefStatement PopFunction()
        //{
        //    if (_functions != null && _functions.Count > 0)
        //    {
        //        return _functions.Pop();
        //    }
        //    return null;
        //}

        //private void PushFunction(DefDefStatement ret)
        //{
        //    if (_functions == null)
        //    {
        //        _functions = new Stack<DefDefStatement>();
        //    }
        //    _functions.Push(ret);
        //}

        private string ReadName()
        {
            NameToken n = PeekToken() as NameToken;
            if (n == null)
            {
                ReportSyntaxError(_lookahead);
            }
            NextToken();
            return n.Name;
        }

        private Statement ParseDecorated()
        {
            throw new NotImplementedException();
        }

        private Statement ParseTryStatement()
        {
            throw new NotImplementedException();
        }

        private Statement ParseForStmt()
        {
            throw new NotImplementedException();
        }

        private Statement ParseWhileStmt()
        {
            throw new NotImplementedException();
        }

        private Statement ParseIfStmt()
        {
            throw new NotImplementedException();
        }

        public ScalaAst ParseTopExpression()
        {
            try
            {
                // TODO: move from source unit  .TrimStart(' ', '\t')
                _globalParent = new ScalaAst("asdf", null);
                ReturnStatement ret = new ReturnStatement(ParseTestListAsExpression());
                ret.SetLoc(_globalParent, 0, 0);
                return FinishParsing(ret);
            }
            catch (BadSourceException bse)
            {
                throw BadSourceError(bse);
            }
        }

        /// <summary>
        /// Given the interactive text input for a compound statement, calculate what the
        /// indentation level of the next line should be
        /// </summary>
        public static int GetNextAutoIndentSize(string text, int autoIndentTabWidth)
        {
            ContractUtils.RequiresNotNull(text, "text");

            Debug.Assert(text[text.Length - 1] == '\n');
            string[] lines = text.Split(newLineChar);
            if (lines.Length <= 1) return 0;
            string lastLine = lines[lines.Length - 2];

            // Figure out the number of white-spaces at the start of the last line
            int startingSpaces = 0;
            while (startingSpaces < lastLine.Length && lastLine[startingSpaces] == ' ')
                startingSpaces++;

            // Assume the same indent as the previous line
            int autoIndentSize = startingSpaces;
            // Increase the indent if this looks like the start of a compounds statement.
            // Ideally, we would ask the parser to tell us the exact indentation level
            if (lastLine.TrimEnd(whiteSpace).EndsWith(":"))
                autoIndentSize += autoIndentTabWidth;

            return autoIndentSize;
        }

        public ErrorSink ErrorSink
        {
            get
            {
                return _errors;
            }
            set
            {
                ContractUtils.RequiresNotNull(value, "value");
                _errors = value;
            }
        }

        public ParserSink ParserSink
        {
            get
            {
                return _sink;
            }
            set
            {
                if (_sink == ParserSink.Null)
                {
                    _sink = null;
                }
                else
                {
                    _sink = value;
                }
            }
        }

        public int ErrorCode
        {
            get { return _errorCode; }
        }

        public void Reset(SourceUnit sourceUnit)
        {
            //ContractUtils.RequiresNotNull(sourceUnit, "sourceUnit");

            _sourceUnit = sourceUnit;
            _token = new TokenWithSpan();
            _lookahead = new TokenWithSpan();
            _privatePrefix = null;

            _parsingStarted = false;
            _errorCode = 0;
        }

        public void Reset()
        {
            Reset(_sourceUnit);
        }

        #endregion

        #region Error Reporting

        private void ReportSyntaxError(TokenWithSpan t)
        {
            ReportSyntaxError(t, ErrorCodes.SyntaxError);
        }

        private void ReportSyntaxError(TokenWithSpan t, int errorCode)
        {
            ReportSyntaxError(t.Token, t.Span, errorCode, true);
        }

        private void ReportSyntaxError(Token t, IndexSpan span, int errorCode, bool allowIncomplete)
        {
            var start = span.Start;
            var end = span.End;

            if (allowIncomplete && (t.Kind == TokenKind.EOF || (_tokenizer.IsEndOfFile)))
            {
                errorCode |= ErrorCodes.IncompleteStatement;
            }

            string msg = String.Format(System.Globalization.CultureInfo.InvariantCulture, GetErrorMessage(t, errorCode), t.Image);

            ReportSyntaxError(start, end, msg, errorCode);
        }

        private static string GetErrorMessage(Token t, int errorCode)
        {
            string msg;
            if ((errorCode & ~ErrorCodes.IncompleteMask) == ErrorCodes.IndentationError)
            {
                msg = "Unexpected Indentation";
            }
            else if (t.Kind != TokenKind.EOF)
            {
                msg = "Unexpected Token";
            }
            else
            {
                msg = "unexpected EOF while parsing";
            }

            return msg;
        }

        private void ReportSyntaxError(string message)
        {
            ReportSyntaxError(_lookahead.Span.Start, _lookahead.Span.End, message);
        }

        internal void ReportSyntaxError(int start, int end, string message)
        {
            ReportSyntaxError(start, end, message, ErrorCodes.SyntaxError);
        }

        internal void ReportSyntaxError(int start, int end, string message, int errorCode)
        {
            // save the first one, the next error codes may be induced errors:
            if (_errorCode == 0)
            {
                _errorCode = errorCode;
            }
            _errors.Add(_sourceUnit,
                message,
                new SourceSpan(_tokenizer.IndexToLocation(start), _tokenizer.IndexToLocation(end)),
                errorCode,
                Severity.FatalError);
        }

        #endregion

        #region Parsing



        #endregion

        #region IDisposable Members

        public void Dispose()
        {
            if (_sourceReader != null)
            {
                _sourceReader.Close();
            }
        }

        #endregion

        #region Implementation Details

        private ScalaAst ParseFileWorker(bool makeModule, bool returnValue)
        {
            _globalParent = new ScalaAst("asdf");
            StartParsing();

            List<Statement> l = new List<Statement>();

            //
            // A future statement must appear near the top of the module. 
            // The only lines that can appear before a future statement are: 
            // - the module docstring (if any), 
            // - comments, 
            // - blank lines, and 
            // - other future statements. 
            // 

            MaybeEatNewLine();

            if (PeekToken(TokenKind.Constant))
            {
                Statement s = ParseStmt();
                l.Add(s);
                //ExpressionStatement es = s as ExpressionStatement;
            }

            MaybeEatNewLine();


            while (true)
            {
                if (MaybeEat(TokenKind.EOF)) break;
                if (MaybeEatNewLine()) continue;

                Statement s = ParseStmt();
                l.Add(s);
            }

            Statement[] stmts = l.ToArray();

            if (returnValue && stmts.Length > 0)
            {
                ExpressionStatement exprStmt = stmts[stmts.Length - 1] as ExpressionStatement;
                if (exprStmt != null)
                {
                    var retStmt = new ReturnStatement(exprStmt.Expression);
                    stmts[stmts.Length - 1] = retStmt;
                    retStmt.SetLoc(_globalParent, exprStmt.Expression.IndexSpan);
                }
            }

            PackageDefStatement ret = new PackageDefStatement("<empty>", l);
            ret.SetLoc(_globalParent, 0, GetEnd());
            return FinishParsing(ret);
        }

        private Statement InternalParseInteractiveInput(out bool parsingMultiLineCmpdStmt, out bool isEmptyStmt)
        {
            try
            {
                Statement s;
                isEmptyStmt = false;
                parsingMultiLineCmpdStmt = false;

                switch (PeekToken().Kind)
                {
                    case TokenKind.NEWLINE:
                        MaybeEatNewLine();
                        Eat(TokenKind.EOF);
                        if (_tokenizer.EndContinues)
                        {
                            parsingMultiLineCmpdStmt = true;
                            _errorCode = ErrorCodes.IncompleteStatement;
                        }
                        else
                        {
                            isEmptyStmt = true;
                        }
                        return null;

                    case TokenKind.IF:
                    case TokenKind.WHILE:
                    case TokenKind.FOR:
                    case TokenKind.TRY:
                    case TokenKind.AT:
                    case TokenKind.DEF:
                    case TokenKind.CLASS:
                    case TokenKind.WITH:
                        parsingMultiLineCmpdStmt = true;
                        s = ParseStmt();
                        EatEndOfInput();
                        break;

                    default:
                        //  parseSimpleStmt takes care of one or more simple_stmts and the Newline
                        s = ParseSimpleStmt();
                        MaybeEatNewLine();
                        Eat(TokenKind.EOF);
                        break;
                }
                return s;
            }
            catch (BadSourceException bse)
            {
                throw BadSourceError(bse);
            }
        }

        private Statement ParseSimpleStmt()
        {
            Statement s = ParseSmallStmt();
            if (MaybeEat(TokenKind.SEMI))
            {
                return s;
            }
            return s;
        }

        private Statement ParseSmallStmt()
        {
            switch (PeekToken().Kind)
            {
                //case TokenKind.KeywordPrint:
                //    return ParsePrintStmt();
                //case TokenKind.KeywordBreak:
                //    if (!_inLoop)
                //    {
                //        ReportSyntaxError("'break' outside loop");
                //    }
                //    return FinishSmallStmt(new BreakStatement());
                //case TokenKind.KeywordContinue:
                //    if (!_inLoop)
                //    {
                //        ReportSyntaxError("'continue' not properly in loop");
                //    }
                //    else if (_inFinally)
                //    {
                //        ReportSyntaxError("'continue' not supported inside 'finally' clause");
                //    }
                //    return FinishSmallStmt(new ContinueStatement());
                case TokenKind.RETURN:
                    return ParseReturnStmt();
                //case TokenKind.KeywordFrom:
                //    return ParseFromImportStmt();
                case TokenKind.IMPORT:
                    return ParseImportStmt();
                //case TokenKind.KeywordGlobal:
                //    return ParseGlobalStmt();
                //case TokenKind.KeywordRaise:
                //    return ParseRaiseStmt();
                //case TokenKind.KeywordAssert:
                //    return ParseAssertStmt();
                //case TokenKind.KeywordExec:
                //    return ParseExecStmt();
                //case TokenKind.KeywordDel:
                //    return ParseDelStmt();
                case TokenKind.YIELD:
                    return ParseYieldStmt();
                case TokenKind.VAL:
                    return ParseValStmt();
                default:
                    return ParseExprStmt();
            }
        }

        private Statement ParseValStmt()
        {
            Eat(TokenKind.VAL);
            var start = GetStart();
            var name = ReadName();
            string valTypeName = default(string);

            if (MaybeEat(TokenKind.COLON))
            {
                valTypeName = ReadName();
            }
            Eat(TokenKind.EQUALS);
            Expression rhs = ParseExpr();
            ValDefStatement ret = new ValDefStatement(ModifierFlags.None, name, rhs);
            if (valTypeName != default(string))
                ret.TypeName = valTypeName.ToString();
            var end = GetEnd();
            ret.SetLoc(_globalParent, start, end);
            return ret;
        }

        private Statement ParseExprStmt()
        {
            var ret = ParseExpression();
            if (ret is ErrorExpression)
            {
                NextToken();
            }

            if (PeekToken(TokenKind.EQUALS))
            {
                return FinishAssignments(ret);
            }
            Statement stmt = new ExpressionStatement(ret);

            return stmt;
        }

        private Statement FinishAssignments(Expression right)
        {
            List<Expression> left = null;
            Expression singleLeft = null;
            while (MaybeEat(TokenKind.EQUALS))
            {
                //string assignError = right.CheckAssign();
                //if (assignError != null)
                //{
                //    ReportSyntaxError(right.StartIndex, right.EndIndex, assignError, ErrorCodes.SyntaxError | ErrorCodes.NoCaret);
                //}

                if (singleLeft == null)
                {
                    singleLeft = right;
                }
                else
                {
                    if (left == null)
                    {
                        left = new List<Expression>();
                        left.Add(singleLeft);
                    }
                    left.Add(right);
                }
                if (MaybeEat(TokenKind.YIELD))
                {

                }
                else
                {
                    right = ParseExpression();
                }
            }

            if (left != null)
            {
                Debug.Assert(left.Count > 0);
                AssignStatement assign = new AssignStatement(left.ToArray(), right);
                assign.SetLoc(_globalParent, left[0].StartIndex, right.EndIndex);
                return assign;
            }
            else
            {
                Debug.Assert(singleLeft != null);
                AssignStatement assign = new AssignStatement(new[] { singleLeft }, right);
                assign.SetLoc(_globalParent, singleLeft.StartIndex, right.EndIndex);
                return assign;
            }
        }

        private Expression ParseExpression()
        {
            Expression ret = ParseExpr();
            return ret;
        }

        private Expression ParseExpr()
        {
            Expression ret = ParsePrimary();
            ret = AddTrailers(ret);
            ret = ParseSelectApply(ret);
            return ret;
        }

        private Expression ParseSelectApply(Expression ret)
        {
            switch (PeekToken().Kind)
            {
                case TokenKind.Name:
                    Expression name = new NameExpression(ReadName());
                    IList<Arg> args;
                    switch (PeekToken().Kind)
                    {
                        case TokenKind.LBRACE:
                            NextToken();
                            args = FinishArgList();
                            break;
                        case TokenKind.Name:
                            NextToken();
                            args = new List<Arg>();
                            args.Add(new Arg(ParsePrimary()));
                            break;
                        default:
                            ReportSyntaxError("The arguments to the functions not passed correctly");
                            args = new List<Arg>();
                            break;
                    }
                    SelectStatement sel = new SelectStatement(ret, name);
                    ApplyExpression app = new ApplyExpression(sel, args);
                    return app;
                default:
                    return ret;
            }
        }

        private Expression AddTrailers(Expression ret)
        {
            while (true)
            {
                switch (PeekToken().Kind)
                {
                    case TokenKind.LPAREN:
                        NextToken();
                        IList<Arg> args = FinishArgList();
                        ApplyExpression call;
                        if (args != null)
                        {
                            call = FinishApplyExpr(ret, args.ToArray());
                        }
                        else
                        {
                            call = new ApplyExpression(ret, new Arg[0]);
                        }

                        call.SetLoc(_globalParent, ret.StartIndex, GetEnd());
                        ret = call;
                        break;
                    case TokenKind.LBRACKET:
                        throw new NotImplementedException();
                    case TokenKind.LBRACE:
                        throw new NotImplementedException();
                    case TokenKind.DOT:
                        NextToken();
                        string name = ReadNameMayBeNone();
                        var nameExpr = new NameExpression(name);
                        SelectStatement fe = new SelectStatement(ret, nameExpr);
                        fe.SetLoc(_globalParent, ret.StartIndex, GetEnd());
                        ret = fe;
                        break;
                    default: return ret;
                }
            }
        }

        private string ReadNameMayBeNone()
        {
            Token t = PeekToken();
            NameToken n = t as NameToken;
            if (n == null)
            {
                ReportSyntaxError("syntax error");
                return null;
            }
            NextToken();
            return n.Name;
        }

        private ApplyExpression FinishApplyExpr(Expression target, params Arg[] args)
        {
            return new ApplyExpression(target, args);
        }

        private IList<Arg> FinishArgList()
        {
            const TokenKind terminator = TokenKind.RPAREN;
            List<Arg> l = new List<Arg>();

            while (true)
            {
                if (MaybeEat(terminator)) { break; }
                var start = GetStart();
                Expression e = ParseExpression();
                Arg a = new Arg(e);
                a.SetLoc(_globalParent, start, GetEnd());
                l.Add(a);
                if (MaybeEat(TokenKind.COMMA))
                {
                    if (_sink != null)
                    {
                        _sink.NextParameter(GetSourceSpan());
                    }
                }
                else { Eat(terminator); break; }
            }
            if (_sink != null) { _sink.EndParameters(GetSourceSpan()); }
            return l;
        }

        private Expression ParsePrimary()
        {
            Token t = PeekToken();
            Expression ret;
            switch (t.Kind)
            {
                //case TokenKind.LPAREN:
                //    NextToken();
                case TokenKind.Name:
                    NextToken();
                    string name = (string)t.Value;
                    ret = new NameExpression(name);
                    if (_sink != null)
                    {
                        _sink.StartName(GetSourceSpan(), name);
                    }
                    //ret = new NameExpression(name);
                    //ret = new ConstantExpression(10);
                    ret.SetLoc(_globalParent, GetStart(), GetEnd());
                    return ret;
                case TokenKind.Constant:
                    NextToken();
                    var start = GetStart();
                    object cv = t.Value;
                    string cvs = cv as string;
                    if (cvs != null)
                    {
                        cv = FinishStringPlus(cvs);
                    }

                    ret = new ConstantExpression(cv);
                    if (cv is int) ret.TypeName = "Int";
                    if (cv is string) ret.TypeName = "String";
                    ret.SetLoc(_globalParent, start, GetEnd());
                    return ret;
                case TokenKind.NEW:
                    NextToken();
                    start = GetStart();
                    ret = ParseNewExpr();
                    ret.SetLoc(_globalParent, start, GetEnd());
                    return ret;
                default:
                    ReportSyntaxError(_lookahead.Token, _lookahead.Span, ErrorCodes.SyntaxError, _allowIncomplete || _tokenizer.EndContinues);
                    ret = new ErrorExpression();
                    ret.SetLoc(_globalParent, _lookahead.Span.Start, _lookahead.Span.End);
                    return ret;
            }
        }

        private Expression ParseNewExpr()
        {
            //Eat(TokenKind.NEW);
            Expression ret = null;
            IList<Arg> args = null;
            var newStart = GetStart();
            bool check = true;
            while (check)
            {
                switch (PeekToken().Kind)
                {
                    case TokenKind.LPAREN:
                        NextToken();
                        args = FinishArgList();
                        break;
                    case TokenKind.Name:
                        string name = (string)PeekToken().Value;
                        var start = GetStart();
                        ret = new NameExpression(name);
                        ret.SetLoc(_globalParent, start, GetEnd());
                        NextToken();
                        break;
                    case TokenKind.DOT:
                        NextToken();
                        start = GetStart();
                        name = (string)PeekToken().Value;
                        var nameExpr = new NameExpression(name);
                        nameExpr.SetLoc(_globalParent, start, GetEnd());
                        SelectStatement fe = new SelectStatement(ret, nameExpr);
                        fe.SetLoc(_globalParent, ret.StartIndex, GetEnd());
                        ret = fe;
                        break;
                    default:
                        check = false;
                        break;
                }
            }
            if (args == null) args = new List<Arg>();
            ret = new NewStatement(ret, args);
            return ret;
        }

        private object FinishStringPlus(string cvs)
        {
            throw new NotImplementedException();
        }

        private Statement ParseYieldStmt()
        {
            throw new NotImplementedException();
        }

        private Statement ParseImportStmt()
        {
            throw new NotImplementedException();
        }

        private Statement ParseReturnStmt()
        {
            throw new NotImplementedException();
        }



        private Expression ParseTestListAsExpression()
        {
            StartParsing();

            Expression expression = ParseTestListAsExpr();
            EatEndOfInput();
            return expression;
        }

        private Expression ParseTestListAsExpr()
        {
            throw new NotImplementedException();
        }

        /// <summary>
        /// Maybe eats a new line token returning true if the token was
        /// eaten.
        /// 
        /// Scala always tokenizes to have only 1  new line character in a 
        /// row.  But we also craete NLToken's and ignore them except for 
        /// error reporting purposes.  This gives us the same errors as 
        /// CScala and also matches the behavior of the standard library 
        /// tokenize module.  This function eats any present NL tokens and throws
        /// them away.
        /// </summary>
        private bool MaybeEatNewLine()
        {
            if (MaybeEat(TokenKind.NEWLINE))
            {
                return true;
            }
            return false;
        }

        /// <summary>
        /// Eats a new line token throwing if the next token isn't a new line.  
        /// 
        /// Scala always tokenizes to have only 1  new line character in a 
        /// row.  But we also craete NLToken's and ignore them except for 
        /// error reporting purposes.  This gives us the same errors as 
        /// CScala and also matches the behavior of the standard library 
        /// tokenize module.  This function eats any present NL tokens and throws
        /// them away.
        /// </summary>
        private bool EatNewLine()
        {
            bool res = Eat(TokenKind.NEWLINE);
            return res;
        }

        private Token EatEndOfInput()
        {
            while (MaybeEatNewLine())
            {
                ;
            }

            Token t = NextToken();
            if (t.Kind != TokenKind.EOF)
            {
                ReportSyntaxError(_token);
            }
            return t;
        }

        private Exception/*!*/ BadSourceError(BadSourceException bse)
        {
            StreamReader sr = _sourceReader.BaseReader as StreamReader;
            if (sr != null && sr.BaseStream.CanSeek)
            {
                //return ScalaContext.ReportEncodingError(sr.BaseStream, _sourceUnit.Path);
                throw new Exception();

            }
            // BUG: We have some weird stream and we can't accurately track the 
            // position where the exception came from.  There are too many levels
            // of buffering below us to re-wind and calculate the actual line number, so
            // we'll give the last line number the tokenizer was at.
            //return ScalaJ.Runtime.Operations.ScalaOps.BadSourceError(
            //    bse._badByte,
            //    new SourceSpan(_tokenizer.CurrentPosition, _tokenizer.CurrentPosition),
            //    _sourceUnit.Path
            //);
            throw new Exception();
        }

        //private bool TrueDivision
        //{
        //    get { return (_languageFeatures & ModuleOptions.TrueDivision) == ModuleOptions.TrueDivision; }
        //}

        //private bool AbsoluteImports
        //{
        //    get { return (_languageFeatures & ModuleOptions.AbsoluteImports) == ModuleOptions.AbsoluteImports; }
        //}

        private void StartParsing()
        {
            if (_parsingStarted)
                throw new InvalidOperationException("Parsing already started. Use Restart to start again.");

            _parsingStarted = true;

            FetchLookahead();
        }

        private int GetEnd()
        {
            Debug.Assert(_token.Token != null, "No token fetched");
            return _token.Span.End;
        }

        private int GetStart()
        {
            Debug.Assert(_token.Token != null, "No token fetched");
            return _token.Span.Start;
        }

        private SourceSpan GetSourceSpan()
        {
            Debug.Assert(_token.Token != null, "No token fetched");
            return new SourceSpan(
                _tokenizer.IndexToLocation(GetStart()),
                _tokenizer.IndexToLocation(GetEnd())
            );
        }

        private Token NextToken()
        {
            _token = _lookahead;
            FetchLookahead();
            return _token.Token;
        }

        private Token PeekToken()
        {
            return _lookahead.Token;
        }

        private void FetchLookahead()
        {
            _lookahead = new TokenWithSpan(_tokenizer.GetNextToken(), _tokenizer.TokenSpan);
        }

        private bool PeekToken(TokenKind kind)
        {
            return PeekToken().Kind == kind;
        }

        private bool PeekToken(Token check)
        {
            return PeekToken() == check;
        }

        private bool Eat(TokenKind kind)
        {
            Token next = PeekToken();
            if (next.Kind != kind)
            {
                ReportSyntaxError(_lookahead);
                return false;
            }
            else
            {
                NextToken();
                return true;
            }
        }

        private bool EatNoEof(TokenKind kind)
        {
            Token next = PeekToken();
            if (next.Kind != kind)
            {
                ReportSyntaxError(_lookahead.Token, _lookahead.Span, ErrorCodes.SyntaxError, false);
                return false;
            }
            NextToken();
            return true;
        }

        private bool MaybeEat(TokenKind kind)
        {
            if (PeekToken().Kind == kind)
            {
                NextToken();
                return true;
            }
            else
            {
                return false;
            }
        }

        private class TokenizerErrorSink : ErrorSink
        {
            private readonly Parser _parser;

            public TokenizerErrorSink(Parser parser)
            {
                Assert.NotNull(parser);
                _parser = parser;
            }

            public override void Add(SourceUnit sourceUnit, string message, SourceSpan span, int errorCode, Severity severity)
            {
                if (_parser._errorCode == 0 && (severity == Severity.Error || severity == Severity.FatalError))
                {
                    _parser._errorCode = errorCode;
                }
                _parser.ErrorSink.Add(sourceUnit, message, span, errorCode, severity);
            }
        }

        #endregion
    }
}
